#!/usr/bin/env python
#coding:utf8

import getopt
import sys
import os
import re
import fnmatch
from xml.etree import ElementTree
from Queue import Queue
from threading import Thread
import threading
import time
import subprocess
import shutil
import hashlib 

lock = threading.Lock()

def usage():
	print sys.argv[0] + "-d project_dir" + "-t time_node"
	print "For example:python source_download.py -d EulerOS_V200R003C00SPC608B113_WN_TOOLS -t \"2018-05-21 05:38:35\""

if not len(sys.argv) == 5:
	usage()
	exit(1)

cachedir_root = sys.argv[2].replace('TOOLS','SOURCE') 
pulgins = ('tar_scm', 'tar_scm_kernel', 'tar_scm_repo', 'tar_scm_kernel_repo', 'tar_scm_kernels_repo','tar_scm_repo_docker')
download_source_status_flag=0
repo_type_dict={'EulerOS_V2R3C00':[['repo','init','-u','ssh://10.175.107.201/EulerOS_V2R3C00_manifest','-m','default_wide.xml','--repo-url=ssh://10.175.107.201/tools/repo','--repo-branch','huawei','--no-repo-verify'],'master'],
'EulerOS_V2R3C00_EulerOS_V2R2C20':[['repo','init','-u','ssh://10.175.107.201/EulerOS_V2R3C00_manifest','-m','EulerOS_V2R2C20_wide.xml','--repo-url=ssh://10.175.107.201/tools/repo','--repo-branch','huawei','--no-repo-verify'],'EulerOS_V2R2C20'],
'EulerOS_V2R3C00_master':[['repo','init','-u','ssh://10.175.107.201/EulerOS_V2R3C00_manifest','-m','default_wide.xml','--repo-url=ssh://10.175.107.201/tools/repo','--repo-branch','huawei','--no-repo-verify'],'master'],
'EulerOS_V2R3C00_next':[['repo','init','-u','ssh://10.175.107.201/EulerOS_V2R3C00_manifest','-m','next_wide.xml','--repo-url=ssh://10.175.107.201/tools/repo','--repo-branch','huawei','--no-repo-verify'],'next'],
'EulerOS_V2R3C00_uvp':[['repo','init','-u','ssh://10.175.107.201/EulerOS_V2R3C00_manifest','-m','uvp_wide.xml','--repo-url=ssh://10.175.107.201/tools/repo','--repo-branch','huawei','--no-repo-verify'],'uvp'],
'EulerOS_V2R3C00_uvp_R5-RC8-CP':[['repo','init','-u','ssh://10.175.107.201/EulerOS_V2R3C00_manifest','-m','uvp_R5-RC8-CP_wide.xml','--repo-url=ssh://10.175.107.201/tools/repo','--repo-branch','huawei','--no-repo-verify'],'uvp_R5-RC8-CP'],
'EulerOS_next_centos7.5':[['repo','init','-u','http://dgggit09-rd.huawei.com/CRDU_RTOS_SDK_GIT/manifest.git','-b','master','-m','EulerOS_next_centos7.5.xml','--no-repo-verify','--repo-branch=huawei','--no-clone-bundle'],'next_centos7.5'],
'EulerOS_next':[['repo','init','-u','http://dgggit09-rd.huawei.com/CRDU_RTOS_SDK_GIT/manifest.git','-b','master','-m','EulerOS_next.xml','--no-repo-verify','--repo-branch=huawei','--no-clone-bundle'],'next'],
'next':[['repo','init','-u','http://dgggit09-rd.huawei.com/euleros/manifest.git','-b','master','-m','EulerOS_next.xml','--no-repo-verify','--repo-branch=huawei','--no-clone-bundle'],'next']}

def safe_run(cmd, cwd, interactive=False):
	env = os.environ.copy()
	env['LANG'] = 'C'
	proc = subprocess.Popen(cmd,
							shell=False,
							stdout=subprocess.PIPE,
							stdin=subprocess.PIPE,
							stderr=subprocess.STDOUT,
							cwd=cwd,
							env=env)
	output = ''
	if interactive:
		stdout_lines = []
		while proc.poll() is None:
			for line in proc.stdout:
				stdout_lines.append(line.rstrip())
		output = '\n'.join(stdout_lines)
	else:
		output = proc.communicate()[0]

	if proc.returncode:
		print 'ERROR(%d): %s' % (proc.returncode, repr(output))
		sys.exit("Command failed(%d): %s" % (proc.returncode, repr(output)))
	return (proc.returncode, output)

class DownloadWorker(Thread):
	def __init__(self,queue):
		Thread.__init__(self)
		self.queue = queue
		self.lock = threading.Lock()

	def run(self):
		while True:
			if self.queue.qsize() > 0:
				(pkginfo) = self.queue.get()
				source_download(pkginfo)
				self.queue.task_done()
			else:
				break

#根据branch、时间、tag切commit
def update_cache_git(scm,dir,time_node,obs_meta_dir,project,package,branch,revision):
	cwd=os.path.join(os.getcwd(),dir)
	if scm == 'repo':
		if obs_meta_dir.endswith('/runc') and (obs_meta_dir.startswith('CRDU_EULEROS_GIT') or \
		obs_meta_dir.startswith('euleros')):
			# 针对V2R7 next_centos7.5分支runc
			obs_meta_dir.replace('/runc','/runc-1.0.0')

		if obs_meta_dir.startswith('EulerOS/'):
			# 针对V2R5 next分支 kernel_aarch64
			obs_meta_dir=obs_meta_dir.replace('EulerOS/','EulerOS_V2R3C00/')
		#针对V2R7 next分支
		obs_meta_dir1=obs_meta_dir.replace('euleros/third_party','CRDU_EULEROS_GIT').\
		replace('euleros/tools','CRDU_EULEROS_GIT/release_repo/build_tools').\
		replace('euleros/platform','CRDU_EULEROS_GIT/partner').replace('euleros/','CRDU_EULEROS_GIT/')
		#针对V2R3 EulerOS_V2R3C00分支 kernel-3.4、kernel-4.1、llt、suse_x86
		obs_meta_dir2=obs_meta_dir.split('/')[0] + "_" + obs_meta_dir.split('/')[-1]
		#针对V2R3 EulerOS_V2R3C00分支其他包
		obs_meta_dir3=obs_meta_dir.split('/')[-1]

	source_info=os.path.join(sys.argv[2],project,".osc","_sources")
	#project目录中的_source文件存在的情况
	if os.path.isfile(source_info):
		with open(source_info,'rt') as f:
			line = f.readline()
			while line:
				#从_source中解析出对应package的branch和tag
				# libsigc++20  两个加号会出现通配符多次匹配,所以将加号更新为减号
				package_name = re.search("name=\"" + package.replace('++','--') + "\"",line.replace('++','--'))
				if scm == 'repo':
					source_obs_meta = re.search("source=\"" + obs_meta_dir.replace('++','--') + "\"",line.replace('++','--')) or \
					re.search("source=\"" + obs_meta_dir1.replace('++','--') + "\"",line.replace('++','--')) or \
					re.search("source=\"" + obs_meta_dir2.replace('++','--') + "\"",line.replace('++','--')) or \
					re.search("source=\"" + obs_meta_dir3.replace('++','--') + "\"",line.replace('++','--'))
				else:
					source_obs_meta = re.search("source=\"" + obs_meta_dir + "\"",line)
				if package_name and source_obs_meta:
					source_branch = line.split(' ')[-3].split('\"')[1]
					source_tag =  line.split(' ')[-2].split('\"')[1]
					#如果从_source中解析出的tag为空字符串，则选择使用时间切commit
					if source_tag == '':
						break
					#切分支
					#针对EulerOS:V2R7:UVP/qemu
					try:
						safe_run(['git', 'checkout',source_branch],cwd,interactive=sys.stdout.isatty())
					except:
						pass
					#切tag
					safe_run(['git', 'checkout',source_tag],cwd,interactive=sys.stdout.isatty())
					return 
				else:
					line = f.readline()


	#project目录中的_source文件不存在的情况,revision为空字符串的情况
	if (revision == '') or (revision is None):
		if branch == None:
			#如果branch为None，默认branch为master，切branch
			safe_run(['git', 'checkout','master'],cwd,interactive=sys.stdout.isatty())
		elif branch in safe_run(['git', 'branch','-a'],cwd,interactive=sys.stdout.isatty())[1]:
			#如果branch在源码的branch列表中，直接切branch
			safe_run(['git', 'checkout',branch],cwd,interactive=sys.stdout.isatty())
		else:
			#如果branch不为None，且不在源码的branch列表中，就认为这个branch是tag，切branch后直接返回
			safe_run(['git', 'checkout',branch],cwd,interactive=sys.stdout.isatty())
			return True
		#根据时间计算最近的commit，切commit
		commit=safe_run(['git','log','--before',time_node,'-1','--format="%H"'],cwd,interactive=sys.stdout.isatty())
		safe_run(['git', 'checkout',commit[1].replace('\"','')],cwd,interactive=sys.stdout.isatty())
	else:
		#revision不为空，去掉revision前后的空格
		revision=revision.strip()
		#更正revision
		if revision == 'UVP':
			revision='uvp'
		#如果revision在源码的branch列表中，就把revision当成branch
		if revision in safe_run(['git', 'branch','-a'],cwd,interactive=sys.stdout.isatty())[1]:
			#切分支
			safe_run(['git', 'checkout',revision],cwd,interactive=sys.stdout.isatty())
			#根据时间计算最近的commit，切commit
			commit=safe_run(['git','log','--before',time_node,'-1','--format="%H"'],cwd,interactive=sys.stdout.isatty())
			safe_run(['git', 'checkout',commit[1].replace('\"','')],cwd,interactive=sys.stdout.isatty())
			
		else:
			#如果revision不为空，切不在源码的branch列表中，就把就把revision当成commit或tag，直接切
			try:
				safe_run(['git', 'checkout',revision],cwd,interactive=sys.stdout.isatty())
			except:
				safe_run(['git', 'checkout',branch],cwd,interactive=sys.stdout.isatty())
				commit=safe_run(['git','log','--before',time_node,'-1','--format="%H"'],cwd,interactive=sys.stdout.isatty())
				safe_run(['git', 'checkout',commit[1].replace('\"','')],cwd,interactive=sys.stdout.isatty())

#计算scm为git或svn的package的hash值
def get_repocache_hash(scm, url, subdir='', revision=''):
	"""Calculate hash fingerprint for repository cache."""
	scm = scm.strip()
	url = url.strip()
	digest = hashlib.new('sha256')
	digest.update(url)
	if scm == 'svn':
		digest.update('/' + subdir)
	if scm == 'git' and revision:
		digest.update('/' + revision)
	return digest.hexdigest()

def iterfindfiles(path, fnexp):
	for root, dirs, files in os.walk(path):
		for filename in fnmatch.filter(files, fnexp):
			yield os.path.join(root, filename)

def repo_source_download(package):
	global lock
	sub_dir=package[2].strip()
	url_dir=package[1].strip()
	repo_type=url_dir.split('/')[0]
	# repo_dir 类似EulerOS_V200R003C00SPC625B001_SOURCE/EulerOS_V2R3C00
	repo_dir=os.path.join(cachedir_root,repo_type)
	# repocachedir 类似EulerOS_V200R003C00SPC625B001_SOURCE/EulerOS_V2R3C00/EulerOS_V2R3C00/suse_x86
	repocachedir=os.path.join(cachedir_root,url_dir)
	# source_url 是package在repo中的下载地址，类似EulerOS_V2R3C00/suse_x86
	source_url=repocachedir.replace(repo_dir + '/','')
	lock.acquire()
	# 初始化 repo
	if not os.path.exists(repo_dir):
		try:
			os.makedirs(repo_dir)
		except:
			if not os.path.isdir(repo_dir):
				lock.release()
				return False
		try:
			# repo init
			safe_run(repo_type_dict[repo_type][0],os.path.join(os.getcwd(),repo_dir),interactive=sys.stdout.isatty())
		except:
			print 'download %-30s %-40s source failure' % (package[4],package[5])
			download_source_status_flag=1
			lock.release()
			return False
	lock.release()

	#如果packge的repocachedir目录不存在，就下载package的代码，如果存在就说明package的代码正在被别的进程下载，或者已经被别的进程下载完毕。
	if not os.path.exists(repocachedir):
		try:
			safe_run(['repo','sync',source_url],os.path.join(os.getcwd(),repo_dir),interactive=sys.stdout.isatty())
			safe_run(['repo','start',repo_type_dict[repo_type][1],'.'],os.path.join(os.getcwd(),repocachedir),interactive=sys.stdout.isatty())
			update_cache_git('repo',repocachedir,package[6],source_url,package[4],package[5],repo_type_dict[repo_type][1],package[3])
		except:
			print 'download %-30s %-40s source failure' % (package[4],package[5])
			download_source_status_flag=1
			return False
	print 'download %-30s %-40s source success' % (package[4],package[5])

def git_source_download(package):
	global lock
	#计算hash值
	hash_dir=get_repocache_hash(package[0],package[1],package[2],package[3])
	#解析sub_dir和url_dir
	sub_dir=package[2].strip()
	url_dir=package[1].strip()
	#gitcachedir是package在SOURCE包中的上级目录
	gitcachedir=os.path.join(cachedir_root,hash_dir)
	#source_dir是package在SOURCE包中的路径
	source_dir=os.path.join(gitcachedir,url_dir.split('/')[-1].replace('.git',''))
	lock.acquire()
	# 如果gitcachedir不存在，那就创建并下载代码,如果已经存在，就说明有其他线程在下载该资源，本线程就可以返回了。
	if not os.path.exists(gitcachedir):
		try:
			os.makedirs(gitcachedir)
		except:
			if not os.path.isdir(gitcachedir):
				lock.release()
				return False
	else:
		lock.release()
		return True
	lock.release()
	try:
		#下载源码
		safe_run(['git','clone',url_dir],os.path.join(os.getcwd(),gitcachedir),interactive=sys.stdout.isatty())
		#切分支和时间点
		update_cache_git('git',source_dir,package[6],url_dir,package[4],package[5],None,package[3])
	except:
		print 'download %-30s %-40s source failure' % (package[4],package[5])
		download_source_status_flag=1
		return False
	print 'download %-30s %-40s source success' % (package[4],package[5])

def update_cache_svn(dir,time_node,url,project,package):
	cwd=os.path.join(os.getcwd(),dir)
	source_info=os.path.join(sys.argv[2],project,".osc","_sources")
	#project目录中的_source文件存在的情况
	if os.path.isfile(source_info):
		with open(source_info,'rt') as f:
			line = f.readline()
			while line:
				#从_source中解析出对应package的tag
				package_name = re.search("name=\"" + package + "\"",line)
				source_url = re.search("source=\"" + url + "\"",line)
				if package_name and source_url:
					tag =  line.split(' ')[-2].split('\"')[1]
					#如果从_source中解析出的tag为空字符串，则选择使用时间切commit
					if tag == '':
						break
					#切tag
					safe_run(['svn','update','-r',tag],cwd,interactive=sys.stdout.isatty())
					return
				else:
					line = f.readline()

	#project目录中的_source文件不存在的情况,则选择使用时间切commit
	safe_run(['svn','update','--force','-r',time_node],cwd,interactive=sys.stdout.isatty())

def svn_source_download(package):
	#计算hash值
	hash_dir=get_repocache_hash(package[0],package[1].strip(),package[2],package[3])
	#解析sub_dir和url_dir
	sub_dir=package[2].strip()
	url_dir=package[1].strip()
	#修改时间格式
	time_info='{' + package[6].replace('-','').replace(':','').replace(' ','T') +'}'
	repocachedir=os.path.join(cachedir_root,hash_dir)
	#source_dir是package在SOURCE包中的路径
	if url_dir.split('/')[-1] == '':
		source_dir=os.path.join(repocachedir,url_dir.split('/')[-2])
	else:
		source_dir=os.path.join(repocachedir,url_dir.split('/')[-1])
	# 如果repocachedir不存在，那就创建并下载代码,如果已经存在，就说明有其他线程在下载该资源，本线程就可以返回了。
	if not os.path.exists(repocachedir):
		try:
			os.makedirs(repocachedir)
		except:
			if not os.path.isdir(repocachedir):
				return False
		try:
			#下载package源码
			safe_run(['svn','checkout',url_dir],os.path.join(os.getcwd(),repocachedir),interactive=sys.stdout.isatty())
			#切tag
			update_cache_svn(source_dir,time_info,url_dir,package[4],package[5])
		except:
			print 'download %-30s %-40s source failure' % (package[4],package[5])
			download_source_status_flag=1
			return False
		print 'download %-30s %-40s source success' % (package[4],package[5])

#根据不同的scm选择不同的源码下载函数
def source_download(pkginfo):
	if pkginfo[0] == 'repo':
		repo_source_download(pkginfo)
	if pkginfo[0] == 'git':
		git_source_download(pkginfo)
	if pkginfo[0] == 'svn':
		svn_source_download(pkginfo)

def get_code_list(project_dir,time_node):
	for filename in iterfindfiles(project_dir, "_service"):
		#遍历整个版本TOOLS目录，寻找_service 文件，过滤掉包含以下内容的_service,其中EulerOS2.5Baseline_20170707、
		#EulerOS2.5BuildTools、EulerOSBaseline_20170720工程是V2R5版本的基线，不需要下载源码。
		if '/.osc/' in filename or '/EulerOS2.5Baseline_20170707/' in filename or '/EulerOS2.5BuildTools/' in \
		filename or '/EulerOSBaseline_20170720/' in filename:
			continue
		project = filename.split('/')[-3]
		package = filename.split('/')[-2]
		with open(filename, 'rt') as f:
			tree = ElementTree.parse(f)
			root = tree.getroot()
			for child in root:
				if child.attrib['name'] not in pulgins:
					continue
				scm = url = None
				subdir = revision = ''
				for child2 in child:
					if child2.attrib['name'] == 'scm':
						scm = child2.text
					if child2.attrib['name'] == 'url':
						url = child2.text
					if child2.attrib['name'] == 'subdir':
						subdir = child2.text
					if child2.attrib['name'] == 'revision':
						revision = child2.text
					if subdir == None:
						subdir = ''
				#解析_service,获取package的信息，包括：scm、url、subdir、revision、project、package、time_node
				pkginfo=[scm,url,subdir,revision,project,package,time_node]
				line = 0
				ret = 0
				if pkginfo[0] == 'svn':
					if pkginfo not in source_code:
						source_code.append(pkginfo)
				# 如果这个包A的subdir为空，且列表不为空，就遍历整个列表，		
				if pkginfo[2] == '' and len(source_code) != 0:
					while (line < len(source_code)):
						#如果某个包B的url、revision和这个包A的url、revision相同，
						#那就删除这个包B，因为包A的源码包含包B的源码，没有必要重复下载。
						if pkginfo[1] == source_code[line][1] and pkginfo[3] == source_code[line][3]:
							del source_code[line]
							line -= 1
						line += 1
					source_code.append(pkginfo)
				#如果包A的subdir不为空，切列表不为空，就遍历整个列表。
				elif (not pkginfo[2] == '') and len(source_code) != 0:
					#如果列表中存在一个包B的url、revision和这个包A的url、revision相同，
					#但是包B的subdir为空，那就不需要往列表中添加包A了，因为包B的源码包含包A的源码。
					while (line < len(source_code)):
						if pkginfo[1] == source_code[line][1] and pkginfo[3] == source_code[line][3] and \
						source_code[line][2] == '':
							ret = 1
							break
						else:
							line += 1
					if ret == 0:
						if pkginfo not in source_code:
							source_code.append(pkginfo)
				else:
					source_code.append(pkginfo)

def main():
	#判断入参
	if not os.path.isdir(sys.argv[2]):
	   print 'Error:' + sys.argv[2] + ' is not a dir'
	   exit(1) 
	#解析入参
	try:
		options,args = getopt.getopt(sys.argv[1:],"d:t:",["dir","time"])
	except getopt.GetoptError:
		sys.exit(1)
	for name,value in options:
		if name in ('-d', '-dir'):
			project_dir = value
		if name in ('-t', '-time'):
			time_node = value
	try:
		time.strptime(time_node,"%Y-%m-%d %H:%M:%S")
	except:
		print "the argv[4] is error,this is time,just like 2018-05-21 05:38:35"
		exit(1)

	#创建3个线程
	queue = Queue()
	threads = 3
	#遍历整个版本TOOLS目录，解析packages的_service文件
	get_code_list(project_dir,time_node)
	#多线程下载源码
	for one_code in source_code:
		queue.put(one_code)

	for x in range(threads):
		worker = DownloadWorker(queue)
		worker.setDaemon(True)
		worker.start()
	queue.join()
	#清理环境
	return download_source_status_flag
source_code = []

if __name__ == '__main__':
   main()
